<!DOCTYPE html>
<html lang="zh-CN">
<head>
  <meta charset="UTF-8">
<meta name="viewport" content="width=device-width, initial-scale=1, maximum-scale=2">
<meta name="theme-color" content="#222">
<meta name="generator" content="Hexo 7.2.0">
  <link rel="apple-touch-icon" sizes="180x180" href="/images/ME.ico">
  <link rel="icon" type="image/png" sizes="32x32" href="/images/ME.ico">
  <link rel="icon" type="image/png" sizes="16x16" href="/images/ME.ico">
  <link rel="mask-icon" href="/images/ME.ico" color="#222">

<link rel="stylesheet" href="/css/main.css">


<link rel="stylesheet" href="/lib/font-awesome/css/all.min.css">

<script id="hexo-configurations">
    var NexT = window.NexT || {};
    var CONFIG = {"hostname":"www.zhoubencheng.com","root":"/","scheme":"Pisces","version":"7.8.0","exturl":false,"sidebar":{"position":"left","display":"post","padding":18,"offset":12,"onmobile":false},"copycode":{"enable":false,"show_result":true,"style":"mac"},"back2top":{"enable":true,"sidebar":false,"scrollpercent":true},"bookmark":{"enable":false,"color":"#222","save":"auto"},"fancybox":false,"mediumzoom":false,"lazyload":false,"pangu":false,"comments":{"style":"tabs","active":null,"storage":true,"lazyload":false,"nav":null},"algolia":{"hits":{"per_page":10},"labels":{"input_placeholder":"Search for Posts","hits_empty":"We didn't find any results for the search: ${query}","hits_stats":"${hits} results found in ${time} ms"}},"localsearch":{"enable":false,"trigger":"auto","top_n_per_article":1,"unescape":false,"preload":false},"motion":{"enable":true,"async":false,"transition":{"post_block":"fadeIn","post_header":"slideDownIn","post_body":"slideDownIn","coll_header":"slideLeftIn","sidebar":"slideUpIn"}}};
  </script>

  <meta name="description" content="pytorch的CI集成架构 如今在大部分ISA指令集中都存在相应的向量加速扩展，例如x86的SSE，PowerPC的VSX，ARM的NEON以及RISCV的RVV扩展。得益于这些向量加速指令，以及相应架构编译器提供的intrinsic函数，pytorch能够在不同的硬件平台上使用不同的向量扩展指令进行硬件层面的加速">
<meta property="og:type" content="article">
<meta property="og:title" content="PytorchCI">
<meta property="og:url" content="http://www.zhoubencheng.com/2024/09/13/PytorchCI/index.html">
<meta property="og:site_name" content="Benson&#39;s blog">
<meta property="og:description" content="pytorch的CI集成架构 如今在大部分ISA指令集中都存在相应的向量加速扩展，例如x86的SSE，PowerPC的VSX，ARM的NEON以及RISCV的RVV扩展。得益于这些向量加速指令，以及相应架构编译器提供的intrinsic函数，pytorch能够在不同的硬件平台上使用不同的向量扩展指令进行硬件层面的加速">
<meta property="og:locale" content="zh_CN">
<meta property="og:image" content="https://i.postimg.cc/fbNxqHGp/build-native.png">
<meta property="og:image" content="https://i.postimg.cc/59gfkprG/Screenshot-from-2024-09-12-15-11-52.png">
<meta property="og:image" content="https://i.postimg.cc/vHzDfff7/Screenshot-from-2024-09-12-16-00-57.png">
<meta property="og:image" content="https://i.postimg.cc/J09NVJ7d/Screenshot-from-2024-09-12-16-41-51.png">
<meta property="og:image" content="https://i.postimg.cc/Vk6JtYX5/CI-FOR-RISCV.png">
<meta property="og:image" content="https://i.postimg.cc/NjSYcKjH/Screenshot-from-2024-09-26-13-55-41.png">
<meta property="og:image" content="https://i.postimg.cc/63hDyqtz/Screenshot-from-2024-09-26-13-57-25.png">
<meta property="og:image" content="https://i.postimg.cc/fb38GvHx/Screenshot-from-2024-09-26-16-14-14.png">
<meta property="article:published_time" content="2024-09-13T02:47:55.000Z">
<meta property="article:modified_time" content="2024-09-27T07:25:55.832Z">
<meta property="article:author" content="Benson">
<meta property="article:tag" content="github">
<meta property="article:tag" content="pytorch">
<meta name="twitter:card" content="summary">
<meta name="twitter:image" content="https://i.postimg.cc/fbNxqHGp/build-native.png">

<link rel="canonical" href="http://www.zhoubencheng.com/2024/09/13/PytorchCI/">


<script id="page-configurations">
  // https://hexo.io/docs/variables.html
  CONFIG.page = {
    sidebar: "",
    isHome : false,
    isPost : true,
    lang   : 'zh-CN'
  };
</script>

  <title>PytorchCI | Benson's blog</title>
  






  <noscript>
  <style>
  .use-motion .brand,
  .use-motion .menu-item,
  .sidebar-inner,
  .use-motion .post-block,
  .use-motion .pagination,
  .use-motion .comments,
  .use-motion .post-header,
  .use-motion .post-body,
  .use-motion .collection-header { opacity: initial; }

  .use-motion .site-title,
  .use-motion .site-subtitle {
    opacity: initial;
    top: initial;
  }

  .use-motion .logo-line-before i { left: initial; }
  .use-motion .logo-line-after i { right: initial; }
  </style>
</noscript>


<style>.github-emoji { position: relative; display: inline-block; width: 1.2em; min-height: 1.2em; overflow: hidden; vertical-align: top; color: transparent; }  .github-emoji > span { position: relative; z-index: 10; }  .github-emoji img, .github-emoji .fancybox { margin: 0 !important; padding: 0 !important; border: none !important; outline: none !important; text-decoration: none !important; user-select: none !important; cursor: auto !important; }  .github-emoji img { height: 1.2em !important; width: 1.2em !important; position: absolute !important; left: 50% !important; top: 50% !important; transform: translate(-50%, -50%) !important; user-select: none !important; cursor: auto !important; } .github-emoji-fallback { color: inherit; } .github-emoji-fallback img { opacity: 0 !important; }</style>
</head>

<body itemscope itemtype="http://schema.org/WebPage">
  <div class="container use-motion">
    <div class="headband"></div>

    <header class="header" itemscope itemtype="http://schema.org/WPHeader">
      <div class="header-inner"><div class="site-brand-container">
  <div class="site-nav-toggle">
    <div class="toggle" aria-label="切换导航栏">
      <span class="toggle-line toggle-line-first"></span>
      <span class="toggle-line toggle-line-middle"></span>
      <span class="toggle-line toggle-line-last"></span>
    </div>
  </div>

  <div class="site-meta">

    <a href="/" class="brand" rel="start">
      <span class="logo-line-before"><i></i></span>
      <h1 class="site-title">Benson's blog</h1>
      <span class="logo-line-after"><i></i></span>
    </a>
      <p class="site-subtitle" itemprop="description">最好的安排</p>
  </div>

  <div class="site-nav-right">
    <div class="toggle popup-trigger">
    </div>
  </div>
</div>




<nav class="site-nav">
  <ul id="menu" class="main-menu menu">
        <li class="menu-item menu-item-home">

    <a href="/" rel="section"><i class="fa fa-home fa-fw"></i>首页</a>

  </li>
        <li class="menu-item menu-item-tags">

    <a href="/tags/" rel="section"><i class="fa fa-tags fa-fw"></i>标签</a>

  </li>
        <li class="menu-item menu-item-categories">

    <a href="/categories/" rel="section"><i class="fa fa-th fa-fw"></i>分类</a>

  </li>
        <li class="menu-item menu-item-archives">

    <a href="/archives/" rel="section"><i class="fa fa-archive fa-fw"></i>归档</a>

  </li>
  </ul>
</nav>




</div>
    </header>

    
  <div class="back-to-top">
    <i class="fa fa-arrow-up"></i>
    <span>0%</span>
  </div>

  <a href="https://github.com/ZhouBencheng" class="github-corner" title="Follow me on GitHub" aria-label="Follow me on GitHub" rel="noopener" target="_blank"><svg width="80" height="80" viewBox="0 0 250 250" aria-hidden="true"><path d="M0,0 L115,115 L130,115 L142,142 L250,250 L250,0 Z"></path><path d="M128.3,109.0 C113.8,99.7 119.0,89.6 119.0,89.6 C122.0,82.7 120.5,78.6 120.5,78.6 C119.2,72.0 123.4,76.3 123.4,76.3 C127.3,80.9 125.5,87.3 125.5,87.3 C122.9,97.6 130.6,101.9 134.4,103.2" fill="currentColor" style="transform-origin: 130px 106px;" class="octo-arm"></path><path d="M115.0,115.0 C114.9,115.1 118.7,116.5 119.8,115.4 L133.7,101.6 C136.9,99.2 139.9,98.4 142.2,98.6 C133.8,88.0 127.5,74.4 143.8,58.0 C148.5,53.4 154.0,51.2 159.7,51.0 C160.3,49.4 163.2,43.6 171.4,40.1 C171.4,40.1 176.1,42.5 178.8,56.2 C183.1,58.6 187.2,61.8 190.9,65.4 C194.5,69.0 197.7,73.2 200.1,77.6 C213.8,80.2 216.3,84.9 216.3,84.9 C212.7,93.1 206.9,96.0 205.4,96.6 C205.1,102.4 203.0,107.8 198.3,112.5 C181.9,128.9 168.3,122.5 157.7,114.1 C157.9,116.9 156.7,120.9 152.7,124.9 L141.0,136.5 C139.8,137.7 141.6,141.9 141.8,141.8 Z" fill="currentColor" class="octo-body"></path></svg></a>


    <main class="main">
      <div class="main-inner">
        <div class="content-wrap">
          

          <div class="content post posts-expand">
            

    
  
  
  <article itemscope itemtype="http://schema.org/Article" class="post-block" lang="zh-CN">
    <link itemprop="mainEntityOfPage" href="http://www.zhoubencheng.com/2024/09/13/PytorchCI/">

    <span hidden itemprop="author" itemscope itemtype="http://schema.org/Person">
      <meta itemprop="image" content="/images/ME.ico">
      <meta itemprop="name" content="Benson">
      <meta itemprop="description" content="Share my thoughts with you">
    </span>

    <span hidden itemprop="publisher" itemscope itemtype="http://schema.org/Organization">
      <meta itemprop="name" content="Benson's blog">
    </span>
      <header class="post-header">
        <h1 class="post-title" itemprop="name headline">
          PytorchCI
        </h1>

        <div class="post-meta">
            <span class="post-meta-item">
              <span class="post-meta-item-icon">
                <i class="far fa-calendar"></i>
              </span>
              <span class="post-meta-item-text">发表于</span>

              <time title="创建时间：2024-09-13 10:47:55" itemprop="dateCreated datePublished" datetime="2024-09-13T10:47:55+08:00">2024-09-13</time>
            </span>
              <span class="post-meta-item">
                <span class="post-meta-item-icon">
                  <i class="far fa-calendar-check"></i>
                </span>
                <span class="post-meta-item-text">更新于</span>
                <time title="修改时间：2024-09-27 15:25:55" itemprop="dateModified" datetime="2024-09-27T15:25:55+08:00">2024-09-27</time>
              </span>
            <span class="post-meta-item">
              <span class="post-meta-item-icon">
                <i class="far fa-folder"></i>
              </span>
              <span class="post-meta-item-text">分类于</span>
                <span itemprop="about" itemscope itemtype="http://schema.org/Thing">
                  <a href="/categories/Utils/" itemprop="url" rel="index"><span itemprop="name">Utils</span></a>
                </span>
            </span>

          
            <span class="post-meta-item" title="阅读次数" id="busuanzi_container_page_pv" style="display: none;">
              <span class="post-meta-item-icon">
                <i class="fa fa-eye"></i>
              </span>
              <span class="post-meta-item-text">阅读次数：</span>
              <span id="busuanzi_value_page_pv"></span>
            </span>

        </div>
      </header>

    
    
    
    <div class="post-body" itemprop="articleBody">

      
        <h2 id="pytorch的ci集成架构">pytorch的CI集成架构</h2>
<p>如今在大部分ISA指令集中都存在相应的向量加速扩展，例如x86的SSE，PowerPC的VSX，ARM的NEON以及RISCV的RVV扩展。得益于这些向量加速指令，以及相应架构编译器提供的<code>intrinsic</code>函数，<code>pytorch</code>能够在不同的硬件平台上使用不同的向量扩展指令进行硬件层面的加速</p>
<span id="more"></span>
<h2 id="sleef的ci测试">SLEEF的CI测试</h2>
<p>以下说明SLEEF库中每一个<code>yaml</code>脚本的CI目的以及作用</p>
<figure class="highlight plaintext"><table><tbody><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br></pre></td><td class="code"><pre><span class="line">.  </span><br><span class="line">├── build_and_test.yml------------在x86-ubuntu中使用qemu-cpu模拟所有</span><br><span class="line">├── build-examples.yml------------在x86-ubuntu平台编译测试./doc/src下的简单样例hellox86.c和tutorial.c</span><br><span class="line">├── build-cross-llvm-mingw.yml----在x86-ubuntu平台上使用llvm-mingw工具链交叉编译出能够运行在arm64-windows和x86-winodws的可执行程序</span><br><span class="line">├── build-as-subproject.yml-------测试SLEE能否作为子项目或者子模块进行编译</span><br><span class="line">├── build-and-test-msys2.yml------测试SLEEF能否在windows-mysy2平台上编译运行</span><br><span class="line">└── build-and-test-macos.yml------测试SLEEF能否在MacOS平台上编译运行</span><br></pre></td></tr></tbody></table></figure>
<p>首先以<code>pytorch</code>调用的第三方仓库<code>SLEEF</code>为例进行CI的简单学习，以<code>SLEEF</code>源代码中的<code>.github/workflow</code>路径下的<a target="_blank" rel="noopener" href="https://github.com/shibatch/sleef/blob/master/.github/workflows/build_and_test.yml"><code>build_and_test.yml</code></a>脚本为例，该脚本在标准的的ubuntu环境中使用不同的<code>qemu-cpu</code>进行测试</p>
<p>工作流<code>build_and_test</code>的job主要分为以下四个</p>
<ol type="1">
<li><p><code>build-native</code>
该工作主要在github官方提供的<code>x86_64</code>服务器上进行<code>SLEEF</code>源码的构建。其步骤是将<code>SLEEF</code>源代码检出到测试环境中；安装相关依赖；并使用<code>cmake</code>指令构建并编译<code>SLEEF</code>源代码；使用<code>upload-artifact@v3</code>action将相关构建文件和编译文件上传暂存到当前工作流中
<a target="_blank" rel="noopener" href="https://postimg.cc/ZWwBCLKx"><img src="https://i.postimg.cc/fbNxqHGp/build-native.png" alt="build-native.png"></a></p></li>
<li><p><code>test-native</code>
该工作主要是在<code>x86_64</code>平台上进行<code>ctest</code>测试。其步骤是检出代码；安装依赖；下载<code>build-native</code>的编译工件；进行<code>ctest</code>
测试；将测试结果通过<code>upload-artifact@v3</code>上传到工作流中 <a target="_blank" rel="noopener" href="https://postimg.cc/5jHcX5bq"><img src="https://i.postimg.cc/59gfkprG/Screenshot-from-2024-09-12-15-11-52.png" alt="Screenshot-from-2024-09-12-15-11-52.png"></a></p></li>
<li><p><code>build-cross</code>
该工作主要使用<em>矩阵策略</em>，使用<code>qemu-cpu</code>进行不同硬件平台和编译器类型的测试。<code>build-cross</code>的编译构建工作大致和<code>build-native</code>相同，需要注意的是多出的两步<code>Check sysroot cache</code>和<code>Create sysroot</code>，前者使用<code>actions/cache@v3</code>检查本地是否存在<code>sysroot</code>目录缓存可用；若本地不存在可用<code>sysroot</code>缓存，则后者使用<code>debootstrap</code>指令创建新的<code>sysroot</code>环境
<a target="_blank" rel="noopener" href="https://postimg.cc/XprV0Z2X"><img src="https://i.postimg.cc/vHzDfff7/Screenshot-from-2024-09-12-16-00-57.png" alt="Screenshot-from-2024-09-12-16-00-57.png"></a></p>
<blockquote>
<p><code>sysroot</code>是一个用于交叉编译和隔离构建环境的重要概念，其作用是提供一个与目标系统环境相似的<strong>根文件系统</strong>，使得编译器能够找到正确的对应目标架构的库、头文件和依赖项</p>
</blockquote>
<p>以下是有关github
action中矩阵策略的学习，在<code>step</code>中可以使用<code>${{ matrix.arch }}-${{ matrix.compiler }}</code>表示不同的硬件架构和编译器类型组合</p>
<figure class="highlight yaml"><table><tbody><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br></pre></td><td class="code"><pre><span class="line"><span class="attr">matrix:</span></span><br><span class="line">     <span class="attr">arch:</span> [<span class="string">aarch64</span>, <span class="string">armhf</span>, <span class="string">ppc64el</span>, <span class="string">s390x</span>, <span class="string">riscv64</span>]</span><br><span class="line">     <span class="attr">compiler:</span> [<span class="string">gcc</span>, <span class="string">llvm</span>]</span><br><span class="line">     <span class="attr">include:</span></span><br><span class="line">       <span class="bullet">-</span> <span class="attr">arch:</span> <span class="string">armhf</span></span><br><span class="line">         <span class="attr">binfmt:</span> <span class="string">arm</span></span><br><span class="line">         <span class="attr">gnupkg:</span> <span class="string">-arm-linux-gnueabihf</span></span><br><span class="line">       <span class="bullet">-</span> <span class="attr">arch:</span> <span class="string">ppc64el</span></span><br><span class="line">         <span class="attr">binfmt:</span> <span class="string">ppc64le</span></span><br><span class="line">         <span class="attr">gnupkg:</span> <span class="string">-powerpc64le-linux-gnu</span></span><br><span class="line">       <span class="bullet">-</span> <span class="attr">arch:</span> <span class="string">aarch64</span></span><br><span class="line">         <span class="attr">debarch:</span> <span class="string">arm64</span></span><br><span class="line">     <span class="attr">exclude:</span></span><br><span class="line">       <span class="comment"># Only GCC trunk supports the RISC-V V intrinsics and https://github.com/riscv-collab/riscv-gnu-toolchain</span></span><br><span class="line">       <span class="comment"># doesn't track a recent enough version yet</span></span><br><span class="line">       <span class="bullet">-</span> <span class="attr">arch:</span> <span class="string">riscv64</span></span><br><span class="line">         <span class="attr">compiler:</span> <span class="string">gcc</span></span><br></pre></td></tr></tbody></table></figure>
<p>在<code>matrix</code>中，<code>include</code>字段用于扩展或覆盖矩阵的组合，在样例代码中，<code>[armhf, *]</code>的矩阵组合会被添加两个字段，即<code>[armhf, *, arm, -arm-linux-gnueabihf]</code>；同时<code>exclude</code>字段也预示着<code>riscv64</code>不会和<code>gcc</code>进行组合</p></li>
<li><p><code>test-cross</code>
<code>Setup QEMU</code>这一步使用<code>docker/setup-qemu-action</code>这一脚本，在工作流中配置不同架构的<code>docker</code>容器，此方法避免手动下载并配置本地<code>QEMU</code>，通过设置<code>QEMU_CPU</code>环境变量，可以告诉<code>QEMU</code>的<code>docker</code>容器模拟哪一种硬件架构。此外，在项目根目录的<code>CMakeLists.txt</code>文件中使用<code>enable_testing()</code>命令，使能项目可以使用<code>ctest</code>进行测试，<code>ctest</code>测试的可执行文件路径在<code>build/bin/*</code>中。这里需要注意的是，在<code>ctest</code>执行二进制文件时，<code>QEMU</code>容器会自动识别并运行非原生架构的二进制文件，从而使得二进制程序的运行透明于用户（看似直接运行于当前平台）
<a target="_blank" rel="noopener" href="https://postimg.cc/gLHX3xZv"><img src="https://i.postimg.cc/J09NVJ7d/Screenshot-from-2024-09-12-16-41-51.png" alt="Screenshot-from-2024-09-12-16-41-51.png"></a></p></li>
</ol>
<h2 id="如何向可移植库添加硬件相关ci以sleef为例">如何向可移植库添加硬件相关CI（以SLEEF为例）</h2>
<h3 id="多硬件平台模拟">多硬件平台模拟</h3>
<p>假设现在需要向SLEEF仓库中添加对一个硬件平台的CI，仅需要修改<code>test_and_build.yml</code>文件，该文件中需要修改的部分是<code>build-cross</code>和<code>test-cross</code>两个步骤，向其中的<code>strategy \ matrix</code>字段添加硬件配置即可，以riscv64为例</p>
<ul>
<li><p><code>build-cross</code>步骤中矩阵策略添加如下内容</p>
<figure class="highlight yaml"><table><tbody><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br></pre></td><td class="code"><pre><span class="line"><span class="attr">strategy:</span></span><br><span class="line">  <span class="attr">fail-fast:</span> <span class="literal">false</span> <span class="comment"># 并行作业中，存在作业失败不影响其他作业继续运行</span></span><br><span class="line">  <span class="attr">matrix:</span></span><br><span class="line">    <span class="attr">arch:</span> [<span class="string">aarch64</span>, <span class="string">armhf</span>, <span class="string">ppc64el</span>, <span class="string">s390x</span>, <span class="string">riscv64</span>] <span class="comment"># 添加riscv64架构</span></span><br><span class="line">    <span class="attr">compiler:</span> [<span class="string">gcc</span>, <span class="string">llvm</span>]</span><br><span class="line">    <span class="attr">include:</span></span><br><span class="line">      <span class="bullet">-</span> <span class="attr">arch:</span> <span class="string">armhf</span></span><br><span class="line">        <span class="attr">binfmt:</span> <span class="string">arm</span></span><br><span class="line">        <span class="attr">gnupkg:</span> <span class="string">-arm-linux-gnueabihf</span></span><br><span class="line">      <span class="bullet">-</span> <span class="attr">arch:</span> <span class="string">ppc64el</span></span><br><span class="line">        <span class="attr">binfmt:</span> <span class="string">ppc64le</span></span><br><span class="line">        <span class="attr">gnupkg:</span> <span class="string">-powerpc64le-linux-gnu</span></span><br><span class="line">      <span class="bullet">-</span> <span class="attr">arch:</span> <span class="string">aarch64</span></span><br><span class="line">        <span class="attr">debarch:</span> <span class="string">arm64</span></span><br><span class="line">    <span class="attr">exclude:</span> <span class="comment"># 此处指出矩阵策略中riscv64不要和gcc组合，因此只有[rsicv64, llvm]组合</span></span><br><span class="line">      <span class="comment"># Only GCC trunk supports the RISC-V V intrinsics and https://github.com/riscv-collab/riscv-gnu-toolchain</span></span><br><span class="line">      <span class="comment"># doesn't track a recent enough version yet</span></span><br><span class="line">      <span class="bullet">-</span> <span class="attr">arch:</span> <span class="string">riscv64</span></span><br><span class="line">        <span class="attr">compiler:</span> <span class="string">gcc</span></span><br></pre></td></tr></tbody></table></figure></li>
<li><p><code>test-cross</code>步骤中矩阵策略添加如下内容</p>
<figure class="highlight yaml"><table><tbody><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br><span class="line">29</span><br><span class="line">30</span><br><span class="line">31</span><br><span class="line">32</span><br><span class="line">33</span><br><span class="line">34</span><br><span class="line">35</span><br><span class="line">36</span><br><span class="line">37</span><br><span class="line">38</span><br><span class="line">39</span><br><span class="line">40</span><br><span class="line">41</span><br><span class="line">42</span><br><span class="line">43</span><br><span class="line">44</span><br><span class="line">45</span><br><span class="line">46</span><br><span class="line">47</span><br><span class="line">48</span><br><span class="line">49</span><br><span class="line">50</span><br><span class="line">51</span><br><span class="line">52</span><br><span class="line">53</span><br><span class="line">54</span><br><span class="line">55</span><br><span class="line">56</span><br><span class="line">57</span><br><span class="line">58</span><br><span class="line">59</span><br><span class="line">60</span><br><span class="line">61</span><br><span class="line">62</span><br><span class="line">63</span><br><span class="line">64</span><br><span class="line">65</span><br></pre></td><td class="code"><pre><span class="line"><span class="attr">strategy:</span></span><br><span class="line">  <span class="attr">fail-fast:</span> <span class="literal">false</span></span><br><span class="line">  <span class="attr">matrix:</span></span><br><span class="line">    <span class="attr">include:</span></span><br><span class="line">      <span class="comment"># AArch64</span></span><br><span class="line">      <span class="bullet">-</span> <span class="attr">arch:</span> <span class="string">aarch64</span></span><br><span class="line">        <span class="attr">compiler:</span> <span class="string">gcc</span></span><br><span class="line">        <span class="attr">qemu_cpu:</span> <span class="string">"max,sve=off"</span></span><br><span class="line">      <span class="bullet">-</span> <span class="attr">arch:</span> <span class="string">aarch64</span></span><br><span class="line">        <span class="attr">compiler:</span> <span class="string">gcc</span></span><br><span class="line">        <span class="attr">qemu_cpu:</span> <span class="string">"max,sve=on,sve128=on"</span></span><br><span class="line">      <span class="bullet">-</span> <span class="attr">arch:</span> <span class="string">aarch64</span></span><br><span class="line">        <span class="attr">compiler:</span> <span class="string">gcc</span></span><br><span class="line">        <span class="attr">qemu_cpu:</span> <span class="string">"max,sve=on,sve256=on"</span></span><br><span class="line">      <span class="bullet">-</span> <span class="attr">arch:</span> <span class="string">aarch64</span></span><br><span class="line">        <span class="attr">compiler:</span> <span class="string">gcc</span></span><br><span class="line">        <span class="attr">qemu_cpu:</span> <span class="string">"max,sve=on,sve512=on"</span></span><br><span class="line">      <span class="bullet">-</span> <span class="attr">arch:</span> <span class="string">aarch64</span></span><br><span class="line">        <span class="attr">compiler:</span> <span class="string">llvm</span></span><br><span class="line">        <span class="attr">qemu_cpu:</span> <span class="string">"max,sve=off"</span></span><br><span class="line">      <span class="bullet">-</span> <span class="attr">arch:</span> <span class="string">aarch64</span></span><br><span class="line">        <span class="attr">compiler:</span> <span class="string">llvm</span></span><br><span class="line">        <span class="attr">qemu_cpu:</span> <span class="string">"max,sve=on,sve128=on"</span></span><br><span class="line">      <span class="bullet">-</span> <span class="attr">arch:</span> <span class="string">aarch64</span></span><br><span class="line">        <span class="attr">compiler:</span> <span class="string">llvm</span></span><br><span class="line">        <span class="attr">qemu_cpu:</span> <span class="string">"max,sve=on,sve256=on"</span></span><br><span class="line">      <span class="bullet">-</span> <span class="attr">arch:</span> <span class="string">aarch64</span></span><br><span class="line">        <span class="attr">compiler:</span> <span class="string">llvm</span></span><br><span class="line">        <span class="attr">qemu_cpu:</span> <span class="string">"max,sve=on,sve512=on"</span></span><br><span class="line">      <span class="comment"># Aarch32</span></span><br><span class="line">      <span class="bullet">-</span> <span class="attr">arch:</span> <span class="string">armhf</span></span><br><span class="line">        <span class="attr">compiler:</span> <span class="string">gcc</span></span><br><span class="line">        <span class="attr">binfmt:</span> <span class="string">arm</span></span><br><span class="line">        <span class="attr">qemu_cpu:</span> <span class="string">"max"</span></span><br><span class="line">      <span class="bullet">-</span> <span class="attr">arch:</span> <span class="string">armhf</span></span><br><span class="line">        <span class="attr">compiler:</span> <span class="string">llvm</span></span><br><span class="line">        <span class="attr">binfmt:</span> <span class="string">arm</span></span><br><span class="line">        <span class="attr">qemu_cpu:</span> <span class="string">"max"</span></span><br><span class="line">      <span class="comment"># PPC64</span></span><br><span class="line">      <span class="bullet">-</span> <span class="attr">arch:</span> <span class="string">ppc64el</span></span><br><span class="line">        <span class="attr">compiler:</span> <span class="string">gcc</span></span><br><span class="line">        <span class="attr">binfmt:</span> <span class="string">ppc64le</span></span><br><span class="line">        <span class="attr">qemu_cpu:</span> <span class="string">"power10"</span></span><br><span class="line">      <span class="bullet">-</span> <span class="attr">arch:</span> <span class="string">ppc64el</span></span><br><span class="line">        <span class="attr">compiler:</span> <span class="string">llvm</span></span><br><span class="line">        <span class="attr">binfmt:</span> <span class="string">ppc64le</span></span><br><span class="line">        <span class="attr">qemu_cpu:</span> <span class="string">"power10"</span></span><br><span class="line">      <span class="comment"># IBM Z</span></span><br><span class="line">      <span class="bullet">-</span> <span class="attr">arch:</span> <span class="string">s390x</span></span><br><span class="line">        <span class="attr">compiler:</span> <span class="string">gcc</span></span><br><span class="line">      <span class="bullet">-</span> <span class="attr">arch:</span> <span class="string">s390x</span></span><br><span class="line">        <span class="attr">compiler:</span> <span class="string">llvm</span></span><br><span class="line">      <span class="comment"># RISC-V</span></span><br><span class="line">      <span class="bullet">-</span> <span class="attr">arch:</span> <span class="string">riscv64</span> <span class="comment"># 此处配置不同的qemu_cpu向量扩展，后续让qemu容器模拟不同向量扩展的硬件平台</span></span><br><span class="line">        <span class="attr">compiler:</span> <span class="string">llvm</span></span><br><span class="line">        <span class="attr">qemu_cpu:</span> <span class="string">"rv64,zba=true,zbb=true,zbs=true,v=false"</span></span><br><span class="line">      <span class="bullet">-</span> <span class="attr">arch:</span> <span class="string">riscv64</span></span><br><span class="line">        <span class="attr">compiler:</span> <span class="string">llvm</span></span><br><span class="line">        <span class="attr">qemu_cpu:</span> <span class="string">"rv64,zba=true,zbb=true,zbs=true,v=true,vlen=128,elen=64,vext_spec=v1.0"</span></span><br><span class="line">      <span class="bullet">-</span> <span class="attr">arch:</span> <span class="string">riscv64</span></span><br><span class="line">        <span class="attr">compiler:</span> <span class="string">llvm</span></span><br><span class="line">        <span class="attr">qemu_cpu:</span> <span class="string">"rv64,zba=true,zbb=true,zbs=true,v=true,vlen=256,elen=64,vext_spec=v1.0"</span></span><br><span class="line">      <span class="bullet">-</span> <span class="attr">arch:</span> <span class="string">riscv64</span></span><br><span class="line">        <span class="attr">compiler:</span> <span class="string">llvm</span></span><br><span class="line">        <span class="attr">qemu_cpu:</span> <span class="string">"rv64,zba=true,zbb=true,zbs=true,v=true,vlen=512,elen=64,vext_spec=v1.0"</span></span><br></pre></td></tr></tbody></table></figure></li>
</ul>
<h3 id="交叉编译-交叉测试">交叉编译 &amp; 交叉测试</h3>
<p>SLEEF的做法是在github
action官方提供的服务器(x86-ubuntu)平台上，使用docker镜像模拟的qemu硬件平台进行测试，借助于<code>sysroot</code>文件系统在本地交叉编译出目标系统的二进制文件后，我们便可以在原生平台上直接运行非原生架构的二进制文件，qemu容器进行会自动识别并运行当前的异构二进制文件。</p>
<p>为此我模仿<code>test_and_build.yml</code>文件编写了特定于RISCV64架构的SLEEF
CI脚本，并附上详细的注释，代码链接<a target="_blank" rel="noopener" href="https://github.com/ZhouBencheng/sleef/blob/master/.github/workflows/build-and-test-riscv64.yml">https://github.com/ZhouBencheng/sleef/blob/master/.github/workflows/build-and-test-riscv64.yml</a></p>
<p>CI结果如下</p>
<p><a target="_blank" rel="noopener" href="https://postimg.cc/JGf7VC6W"><img src="https://i.postimg.cc/Vk6JtYX5/CI-FOR-RISCV.png" alt="CI-FOR-RISCV.png"></a></p>
<h2 id="pytorch的ci测试">pytorch的CI测试</h2>
<p>以<code>aarch64</code>架构的<code>SVE2</code>向量扩展为例，首先观察<code>pytorch</code>添加<code>SVE2</code>扩展的代码触发而<code>RVV</code>未能触发的CI有哪些</p>
<p>关于添加<code>SVE2</code>支持会触发的工作流</p>
<ul>
<li><p><code>check labels</code></p>
<p>该工作流主要调用<code>.github/script/check_labels.py</code>脚本对本次<code>pr</code>进行标签测试，检查本次<code>pr</code>是否包含应该含有的标签</p>
<hr></li>
<li><p><code>linux-binary-libtorch-pre-cxx11</code></p>
<p>在该工作流的主要步骤是<code>libtorch-cpu-shared-with-deps-cxx11-build</code>，其内容是使用<code>.github/workflow/_binary-build-linux.yml</code>脚本构建出<code>libtorch</code>共享库及其所有依赖库；在<code>libtorch-cpu-shared-with-deps-cxx11-test</code>步骤中使用<code>.github/workflow/_binary-test-linux.yml</code>脚本进行二进制文件的测试
<a target="_blank" rel="noopener" href="https://postimg.cc/BjTR5QDS"><img src="https://i.postimg.cc/NjSYcKjH/Screenshot-from-2024-09-26-13-55-41.png" alt="Screenshot-from-2024-09-26-13-55-41.png"></a></p>
<hr></li>
<li><p><code>linux-binary-libtorch-cxx11-abi</code></p>
<p><a target="_blank" rel="noopener" href="https://postimg.cc/WFh9y2DJ"><img src="https://i.postimg.cc/63hDyqtz/Screenshot-from-2024-09-26-13-57-25.png" alt="Screenshot-from-2024-09-26-13-57-25.png"></a>
<code>linux-binary-libtorch-pre-cxx11</code>和<code>linux-binary-libtorch-cxx11-abi</code>两个工作流的内容非常相似但目的不同，其中
<code>linux-binary-libtorch-pre-cxx11</code>用于构建编译<code>libtorch</code>的<code>C++11</code>之前的版本，旨在让<code>libtorch</code>能够向后兼容未启用<code>C++11</code>标准老旧系统；而<code>linux-binary-libtorch-cxx11-abi</code>在构建编译<code>libtorch</code>时开启<code>C++11</code>新特性，适用于开启了<code>C++11</code>特性新标准环境。注意在前者中，<code>DESIRED_DEVTOOLSET</code>被设置为<code>pre-cxx11</code>，指明使用旧的开发工具链；而后者的<code>DESIRED_DEVTOOLSET</code>设置为<code>cxx11-abi</code>，明确启用<code>C++11</code>的二进制接口</p>
<hr></li>
<li><p><code>linux-binary-manywheel</code></p>
<p>同上面两个工作流，本工作流同样调用<code>_binary-build-linux.yml</code>和<code>_binary-test-linux.yml</code>两个脚本，但是分别在<code>py3.9-cuda11.8</code>
<code>py3.9-cuda12.1</code>
<code>py3.9-cuda12.4</code>几个平台上进行构建测试</p>
<hr></li>
<li><p><code>linux-binary-manywheel-split</code></p>
<p>该工作流关于<code>libtorch</code>构建和<code>linux-binary-manywheel</code>基本相同，但该工作流注重<code>split build</code>，<code>_binary-build-linux</code>关于<code>split build</code>的描述如下</p>
<blockquote>
<p>[Experimental] Build a libtorch only wheel and build pytorch such
that are built from the libtorch wheel.</p>
</blockquote>
<p>在传统<code>pytorch</code>构建中，<code>libtorch</code>和<code>pytorch</code>通常会一起构建，采用<code>split build</code>可以先独立构建<code>libtorch</code>，将<code>libtorch</code>作为单独一个组件进行分发和使用，这种方法对纯<code>C++</code>项目的开发非常有帮助，避免不必要的<code>python</code>依赖。其次开发者可以基于已经构建好的<code>libtorch</code>轮子构建<code>pytorch</code>，增强兼容性和稳定性</p>
<hr></li>
<li><p><code>linux-aarch64</code></p>
<p>本工作流中，在build阶段调用<code>.github/workflow/_linux-build.yml</code>脚本，并设置要拉取的docker镜像为<code>pytorch-linux-jammy-aarch64-py3.10-gcc11</code>。在<code>_linux-build.yml</code>脚本中，该工作主要在<code>pytorch</code>预先配置好的docker容器中运行<code>build.sh</code>进行构建。而在test阶段，调用<code>.github/workflow/_linux-test.yml</code>脚本，在改脚本中使用预先配置好的docker容器运行<code>.ci/onnx/test.sh</code>或<code>.ci/pytorch/test.sh</code>脚本进行测试</p>
<hr></li>
<li><p><code>trunk</code> <code>periodic</code></p>
<p>这两个工作流都在<code>linux-jammy</code> <code>macos</code>
<code>windows</code>三个平台上进行构建测试，并通过设置<code>cron</code>表达式，在特定的时间点定时运行相关CI</p>
<hr></li>
</ul>
<p><strong>注意上述几个<code>binary</code>相关的工作流都由<code>Jinja</code>模板<code>linux_binary_build_workflow.j2</code>文件动态生成</strong></p>
<p>关于<code>linux-binary-libtorch-pre-cxx11</code>
<code>linux-binary-libtorch-cxx11-abi</code>
<code>linux-binary-manywheel</code>工作流能够触发是因为相关<a target="_blank" rel="noopener" href="https://github.com/pytorch/pytorch/pull/119571">PR</a>具备标签<code>ciflow/trunk</code>，<code>periodic</code>
<code>linux-binary-manywheel-split</code>工作流能够触发是因为具备标签<code>ciflow/periodic</code>。其中<strong>硬件相关</strong>的CI<code>linux-aarch64</code>能够出发是因为具备标签<code>ciflow-aarch64</code>。</p>
<p>通过检查github
pr标签中的集合，发现<code>ciflow</code>标签下与ISA相关的特定标签仅有<code>ciflow-aarch64</code>一个，以其他<a target="_blank" rel="noopener" href="https://github.com/pytorch/pytorch/pull/132746/">PR</a>——ppc64le硬件的vsx平扩展为例，一方面<code>ppcle64</code>硬件相关的PR不存在类似<code>ciflow/ppc64le</code>这样的标签，再者其触发的工作流也没有硬件相关CI，CI如下，是<code>SVE</code>触发CI的子集</p>
<p><a target="_blank" rel="noopener" href="https://postimg.cc/qtTcn2HR"><img src="https://i.postimg.cc/fb38GvHx/Screenshot-from-2024-09-26-16-14-14.png" alt="Screenshot-from-2024-09-26-16-14-14.png"></a></p>

    </div>

    
    
    

      <footer class="post-footer">
          
          <div class="post-tags">
              <a href="/tags/github/" rel="tag"><i class="fa fa-tag"></i> github</a>
              <a href="/tags/pytorch/" rel="tag"><i class="fa fa-tag"></i> pytorch</a>
          </div>

        


        
    <div class="post-nav">
      <div class="post-nav-item">
    <a href="/2024/09/11/CMU15445-Project/" rel="prev" title="CMU15445 Project">
      <i class="fa fa-chevron-left"></i> CMU15445 Project
    </a></div>
      <div class="post-nav-item">
    <a href="/2024/12/09/linux%E9%85%8D%E7%BD%AEclash%E4%BB%A3%E7%90%86/" rel="next" title="linux配置clash代理">
      linux配置clash代理 <i class="fa fa-chevron-right"></i>
    </a></div>
    </div>
      </footer>
    
  </article>
  
  
  



          </div>
          

<script>
  window.addEventListener('tabs:register', () => {
    let { activeClass } = CONFIG.comments;
    if (CONFIG.comments.storage) {
      activeClass = localStorage.getItem('comments_active') || activeClass;
    }
    if (activeClass) {
      let activeTab = document.querySelector(`a[href="#comment-${activeClass}"]`);
      if (activeTab) {
        activeTab.click();
      }
    }
  });
  if (CONFIG.comments.storage) {
    window.addEventListener('tabs:click', event => {
      if (!event.target.matches('.tabs-comment .tab-content .tab-pane')) return;
      let commentClass = event.target.classList[1];
      localStorage.setItem('comments_active', commentClass);
    });
  }
</script>

        </div>
          
  
  <div class="toggle sidebar-toggle">
    <span class="toggle-line toggle-line-first"></span>
    <span class="toggle-line toggle-line-middle"></span>
    <span class="toggle-line toggle-line-last"></span>
  </div>

  <aside class="sidebar">
    <div class="sidebar-inner">

      <ul class="sidebar-nav motion-element">
        <li class="sidebar-nav-toc">
          文章目录
        </li>
        <li class="sidebar-nav-overview">
          站点概览
        </li>
      </ul>

      <!--noindex-->
      <div class="post-toc-wrap sidebar-panel">
          <div class="post-toc motion-element"><ol class="nav"><li class="nav-item nav-level-2"><a class="nav-link" href="#pytorch%E7%9A%84ci%E9%9B%86%E6%88%90%E6%9E%B6%E6%9E%84"><span class="nav-number">1.</span> <span class="nav-text">pytorch的CI集成架构</span></a></li><li class="nav-item nav-level-2"><a class="nav-link" href="#sleef%E7%9A%84ci%E6%B5%8B%E8%AF%95"><span class="nav-number">2.</span> <span class="nav-text">SLEEF的CI测试</span></a></li><li class="nav-item nav-level-2"><a class="nav-link" href="#%E5%A6%82%E4%BD%95%E5%90%91%E5%8F%AF%E7%A7%BB%E6%A4%8D%E5%BA%93%E6%B7%BB%E5%8A%A0%E7%A1%AC%E4%BB%B6%E7%9B%B8%E5%85%B3ci%E4%BB%A5sleef%E4%B8%BA%E4%BE%8B"><span class="nav-number">3.</span> <span class="nav-text">如何向可移植库添加硬件相关CI（以SLEEF为例）</span></a><ol class="nav-child"><li class="nav-item nav-level-3"><a class="nav-link" href="#%E5%A4%9A%E7%A1%AC%E4%BB%B6%E5%B9%B3%E5%8F%B0%E6%A8%A1%E6%8B%9F"><span class="nav-number">3.1.</span> <span class="nav-text">多硬件平台模拟</span></a></li><li class="nav-item nav-level-3"><a class="nav-link" href="#%E4%BA%A4%E5%8F%89%E7%BC%96%E8%AF%91-%E4%BA%A4%E5%8F%89%E6%B5%8B%E8%AF%95"><span class="nav-number">3.2.</span> <span class="nav-text">交叉编译 &amp; 交叉测试</span></a></li></ol></li><li class="nav-item nav-level-2"><a class="nav-link" href="#pytorch%E7%9A%84ci%E6%B5%8B%E8%AF%95"><span class="nav-number">4.</span> <span class="nav-text">pytorch的CI测试</span></a></li></ol></div>
      </div>
      <!--/noindex-->

      <div class="site-overview-wrap sidebar-panel">
        <div class="site-author motion-element" itemprop="author" itemscope itemtype="http://schema.org/Person">
    <img class="site-author-image" itemprop="image" alt="Benson"
      src="/images/ME.ico">
  <p class="site-author-name" itemprop="name">Benson</p>
  <div class="site-description" itemprop="description">Share my thoughts with you</div>
</div>
<div class="site-state-wrap motion-element">
  <nav class="site-state">
      <div class="site-state-item site-state-posts">
          <a href="/archives/">
        
          <span class="site-state-item-count">17</span>
          <span class="site-state-item-name">日志</span>
        </a>
      </div>
      <div class="site-state-item site-state-categories">
            <a href="/categories/">
          
        <span class="site-state-item-count">7</span>
        <span class="site-state-item-name">分类</span></a>
      </div>
      <div class="site-state-item site-state-tags">
            <a href="/tags/">
          
        <span class="site-state-item-count">25</span>
        <span class="site-state-item-name">标签</span></a>
      </div>
  </nav>
</div>
  <div class="links-of-author motion-element">
      <span class="links-of-author-item">
        <a href="https://github.com/ZhouBencheng" title="GitHub → https:&#x2F;&#x2F;github.com&#x2F;ZhouBencheng" rel="noopener" target="_blank"><i class="fab fa-github fa-fw"></i>GitHub</a>
      </span>
      <span class="links-of-author-item">
        <a href="mailto:zhoubencheng2022@163.com" title="E-Mail → mailto:zhoubencheng2022@163.com" rel="noopener" target="_blank"><i class="fa fa-envelope fa-fw"></i>E-Mail</a>
      </span>
  </div>



      </div>

    </div>
  </aside>
  <div id="sidebar-dimmer"></div>


      </div>
    </main>

    <footer class="footer">
      <div class="footer-inner">
        

        

<div class="copyright">
  
  &copy; 
  <span itemprop="copyrightYear">2024</span>
  <span class="with-love">
    <i class="fa fa-heart"></i>
  </span>
  <span class="author" itemprop="copyrightHolder">Benson</span>
</div>

<!--
  <div class="powered-by">由 <a href="https://hexo.io/" class="theme-link" rel="noopener" target="_blank">Hexo</a> & <a href="https://pisces.theme-next.org/" class="theme-link" rel="noopener" target="_blank">NexT.Pisces</a> 强力驱动
  </div>
-->

        
<div class="busuanzi-count">
  <script async src="https://busuanzi.ibruce.info/busuanzi/2.3/busuanzi.pure.mini.js"></script>
    <span class="post-meta-item" id="busuanzi_container_site_uv" style="display: none;">
      <span class="post-meta-item-icon">
        <i class="fa fa-user"></i>
      </span>
      <span class="site-uv" title="总访客量">
        <span id="busuanzi_value_site_uv"></span>
      </span>
    </span>
    <span class="post-meta-divider">|</span>
    <span class="post-meta-item" id="busuanzi_container_site_pv" style="display: none;">
      <span class="post-meta-item-icon">
        <i class="fa fa-eye"></i>
      </span>
      <span class="site-pv" title="总访问量">
        <span id="busuanzi_value_site_pv"></span>
      </span>
    </span>
</div>








      </div>
    </footer>
  </div>

  
  <script src="/lib/anime.min.js"></script>
  <script src="/lib/velocity/velocity.min.js"></script>
  <script src="/lib/velocity/velocity.ui.min.js"></script>

<script src="/js/utils.js"></script>

<script src="/js/motion.js"></script>


<script src="/js/schemes/pisces.js"></script>


<script src="/js/next-boot.js"></script>


  <script defer src="/lib/three/three.min.js"></script>
    <script defer src="/lib/three/canvas_lines.min.js"></script>


  













<script>
if (document.querySelectorAll('pre.mermaid').length) {
  NexT.utils.getScript('//cdn.jsdelivr.net/npm/mermaid@8/dist/mermaid.min.js', () => {
    mermaid.initialize({
      theme    : 'dark',
      logLevel : 3,
      flowchart: { curve     : 'linear' },
      gantt    : { axisFormat: '%m/%d/%Y' },
      sequence : { actorMargin: 50 }
    });
  }, window.mermaid);
}
</script>


  

  

  

</body>
</html>
